#!/usr/bin/env python
# encoding: utf-8

import json
from source.Lib import Tools
from psycopg2.extras import Json


def ToDataBase():
    lStockInfos = list()

    # 读取原始数据
    with open('/data/stock.json', 'r') as f:
        for line in f.readlines():
            if "code" in line:
                if line[-2] == ",":
                    line = line[:-2]
                try:
                    lStockInfos.append(json.loads(line))
                except:
                    print(line)
    print("加载数量：", len(lStockInfos))

    # 剔除无效数据，数据格式转换Double
    lStockInfos = ETL(lStockInfos)

    cur = Tools.conn.cursor()

    sFinish = set()
    insertCodes = set()
    for item in lStockInfos:
        cur.execute(f"insert into t_report values ('{item[0]}', '{item[3]}', {Json(item[4])})")
        insertCodes.add(item[0])
        if item[0] not in sFinish and len(item[2]):
            cur.execute(f"insert into t_stockinfo values ('{item[0]}', '{item[1]}', '{item[5]}', array{item[2]})")
            sFinish.add(item[0])
    Tools.conn.commit()
    cur.close()
    json.dump(list(insertCodes), open("./stock_filter.json", 'w'))


def ETL(lStockInfos):
    lReturns = list()
    dtVaildFieldCount = dict()
    sVailedField = set()

    for item in lStockInfos:
        for date in item['financial']:
            for k, v in item['financial'][date].items():
                if "--" not in v and len(v) > 1:
                    dtVaildFieldCount.setdefault(k, 0)
                    dtVaildFieldCount[k] += 1
    iValidLimit = len(lStockInfos)/10
    for field, count in dtVaildFieldCount.items():
        if count >= 0:  # iValidLimit
            sVailedField.add(field)

    allFinish = set()
    for item in lStockInfos:
        for date in item['financial']:
            if (item['code'], date) in allFinish:
                continue
            allFinish.add((item['code'], date))
            dtFinancial = dict()
            for k, v in item['financial'][date].items():
                if "--" not in v and len(v) > 1 and k in sVailedField:
                    try:
                        dtFinancial[k] = float(v.replace(',', '').replace("元", ""))
                    except (Exception, ) as e:
                        print(e, k, v)
                        continue

            lReturns.append([item['code'], item['name'], item['profession'], date, dtFinancial, item['market']])
    return lReturns


if __name__ == "__main__":
    ToDataBase()
